import scrapy
from bs4 import BeautifulSoup
#  编写爬虫程序的文件
class BaiduSpider(scrapy.Spider):
    name = "baidu"
    allowed_domains = ["www.baidu.com"]
    # url队列  获取数据时从队列中依次获取url地址   项目中默认组件    调用度器   下载器 request   response
    start_urls = ["https://www.baidu.com"]

    # http请求的结果会自动传入response  requests.get(url,headers)
    def parse(self, response):
        # 获取响应结果
        print(response.url)
        print(response.text)
        soup = BeautifulSoup(response.text, 'lxml')
        title = soup.title.string
        print('title:', title)
